"""Merge files in a directory into a file"""
import argparse
import io
import os


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("in_dir")
    parser.add_argument("out_fn")
    args = parser.parse_args()

    data_root = args.in_dir
    data_files = [os.path.join(data_root, f) for f in os.listdir(data_root)]

    merged_fn = args.out_fn
    out_fn = os.path.join(data_root, merged_fn)
    out_f = io.open(out_fn, "w", encoding="utf-8")

    cnt = 0
    for f in data_files:
        in_f = io.open(f, encoding="utf-8")
        for line in in_f:
            line = line.strip()
            if len(line) > 0:
                out_f.write(line + "\n")
                cnt += 1
                if cnt % 100000 == 0:
                    print(cnt)

    print(cnt)


